package colly

import (
	"Info/dao/mysql"
	"Info/model"
	"Info/utils/Timeutils"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"github.com/gocolly/colly"
	"github.com/gocolly/colly/extensions"
	"log"
	"strings"
)

const baseUrl = "http://dzxx.cssf.cn"

func CollyData() {
	// 创建采集器对象
	collector := colly.NewCollector(func(collector *colly.Collector) {
		extensions.RandomUserAgent(collector)
	})
	// 发起请求之前调用
	collector.OnRequest(func(request *colly.Request) {
		fmt.Println("start visit: ", request.URL.String())
	})
	// 请求期间发生错误,则调用
	collector.OnError(func(response *colly.Response, err error) {
		fmt.Println("请求期间发生错误,则调用:", err)
	})
	// 收到响应后调用
	collector.OnResponse(func(response *colly.Response) {
		fmt.Println("收到响应后调用:", response.Request.URL)
	})
	//OnResponse如果收到的内容是HTML ,则在之后调用
	//解析列表
	collector.OnHTML("div.column", func(e *colly.HTMLElement) {
		// 依次遍历所有的li节点
		e.DOM.Find("li").Each(func(i int, selection *goquery.Selection) {
			href, found := selection.Find("p > a").Attr("href")
			// 如果找到了详情页，则继续下一步的处理
			// 找未在数据库中的数据，比较发布时间
			Date := selection.Find("span").Text()
			currentDate := Timeutils.StringToTimeYMD(Date)
			lastDate, err := mysql.CollegeNews.QueryLastTime()
			if err != nil {
				return
			}
			if Timeutils.IsTime1AfterTime2(currentDate, lastDate) {
				found = true
			} else {
				found = false
			}
			if found {
				parseDetail(collector, href)
			}
		})
	})

	// 查找下一页 > span.pagebox_next
	collector.OnHTML("span.pagebox_next", func(element *colly.HTMLElement) {
		href, found := element.DOM.Find("a").Attr("onclick")
		// 如果有下一页，则继续访问
		if found {
			length := len(href)
			href = href[15 : length-1]
			err := element.Request.Visit(element.Request.AbsoluteURL(href))
			if err != nil {
				return
			}
		}
	})

	// url：请求具体的地址
	err := collector.Visit(baseUrl + "/p208/xydt.html")
	if err != nil {
		fmt.Println("具体错误:", err)
	}
}

func parseDetail(collector *colly.Collector, url string) {
	news := &model.CollegeNews{}

	collector = collector.Clone()
	collector.OnRequest(func(request *colly.Request) {
		log.Println("start visit: ", request.URL.String())
	})

	// 解析详情页数据
	collector.OnResponse(func(response *colly.Response) {
		fmt.Println("收到响应2后调用:", response.Request.URL)
		dataList1 := strings.Split(string(response.Body), "<div class=\"content\">")
		dataList2 := strings.Split(dataList1[1], "<!--尾部-->")
		dataList3 := strings.Split(dataList2[0], "</div>")
		dataTemp := strings.Join(dataList3[:len(dataList3)-3], "</div>")
		dataTemp = "<div class=\"content\">" + dataTemp
		dataList4 := strings.Split(dataTemp, "src=\"")
		data := strings.Join(dataList4, "src=\""+baseUrl)
		news.NewsContent = data
	})

	collector.OnHTML("div[class=column] > .caption", func(element *colly.HTMLElement) {
		title := element.ChildText("h3")
		newsTime := element.ChildText("p > span")
		newsTimeList := strings.Split(newsTime, "间：")
		NewsTitle := title
		NewsDate := Timeutils.StringToTimeYMD(newsTimeList[len(newsTimeList)-1])
		news.NewsTitle = NewsTitle
		news.NewsDate = NewsDate
	})

	url = baseUrl + url
	if err := collector.Visit(url); err != nil {
		return
	}
	if err := mysql.CollegeNews.Insert(news); err != nil {
		return
	}

}
